package com.MyApp.CompanyRank;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.Reader;
import java.io.LineNumberReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;

import org.wikipedia.miner.util.WikiDB;

import au.com.bytecode.opencsv.CSVReader;

import java.math.BigDecimal;

public class PeopleDB {
	public ArrayList<PersonEntry> personList;
	public HashMap<String, PersonEntry> name2person = new HashMap<String, PersonEntry>(320000);
	public WikiDB wikiDB;

	static public String dataDir = "rankingdata" + File.separator +"peopleranking";
	static public String personCSV = dataDir + File.separator + "Person.csv";
	static public String peopleRankingCSV = dataDir + File.separator + "peopleranking.csv";
	static public String almamaterPrimitiveCSV = dataDir + File.separator + "AlmamaterPrimitiveList.csv";

	/** the almamater count */
	static HashMap<String, Integer> almaMaterSumCount;
	static HashMap<String, Integer> almaMaterCount;

	/** the institution count referred by the person list */
	static HashMap<String, Integer> institutionSumCount;
	static HashMap<String, Integer> institutionCount;

	static int maxSum;
	static int maxCount;

	static DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

	public static String[] schemaRow;

	public PeopleDB() throws Exception {
		wikiDB = // WikiDB.getDB(WikiDB.DBType.WIKIMINERDB);
				WikiDB.getDB(WikiDB.DBType.OLDWIKIMINERDB);	
//		WikiDB.getDB(WikiDB.DBType.WIKISQLDB);
		System.out.println(dateFormat.format(new Date()) + " starting wiki PeopleDB init ...");

		System.out.println(dateFormat.format(new Date()) + " reading Person.csv ...");

		personList = readPersonList(wikiDB, personCSV);
		System.out.println(dateFormat.format(new Date()) + " finished reading Person.csv. There are "
				+ personList.size() + " people in the database");
		for (int i = 0; i < personList.size(); i++) {
			PersonEntry p = personList.get(i);
			name2person.put(p.a, p);
		}

		System.out.println(dateFormat.format(new Date()) + " finished building namemap.");
	}

	public void close() {
		wikiDB.close();
	}

	public static void main(String args[]) throws Exception {
		PeopleDB peopleDB = new PeopleDB();

		System.out.println(dateFormat.format(new Date()) + " sorting...");
		Collections.sort(peopleDB.personList, new PersonComparator());
		System.out.println("printing Person sorting results..");
//		printRankingDataInCSV(peopleDB.personList, peopleRankingCSV);

		System.out.println(dateFormat.format(new Date())
				+ " finished person ranking result printing, cumulating almamater and institution stat ...");
		schoolStat(peopleDB.personList);

		System.out.println("building primitive almamater list");
		ArrayList<UEntry> ulist = new ArrayList<UEntry>();
		Iterator it = almaMaterSumCount.entrySet().iterator();
		while (it.hasNext()) {
			Map.Entry pairs = (Map.Entry) it.next();
			Integer v = (Integer) pairs.getValue();
			ulist.add(new UEntry((String) pairs.getKey(), (double) v.intValue()));
		}
		System.out.println("sorting primitive almamater list");
		Collections.sort(ulist);
		System.out.println("printing primitive almamater list ...");
		printAlmamater(ulist, almamaterPrimitiveCSV);

		System.out.println("building primitive institution list");
		ulist = new ArrayList<UEntry>();
		it = institutionSumCount.entrySet().iterator();
		while (it.hasNext()) {
			Map.Entry pairs = (Map.Entry) it.next();
			Integer v = (Integer) pairs.getValue();
			ulist.add(new UEntry((String) pairs.getKey(), (double) v.intValue()));
		}
		System.out.println("sorting primitive institution list");
		Collections.sort(ulist);
		System.out.println("printing primitive institution list ...");
		printInstitution(ulist, dataDir + File.separator + "InstitutionPrimitiveList.csv");

		String rankingDir = "rankingdata";
		String enfile = rankingDir + File.separator + "GlobalList.txt";
		String zhfile = rankingDir + File.separator + "GlobalListcn.txt";
		HashMap<String, String> nameMap = UniversityData.buildEn2ZhMap(enfile, zhfile);
		ulist.clear();

		ArrayList<UEntry> noNameList = new ArrayList<UEntry>();
		HashMap<String, String> parentMap = new HashMap<String, String>();
		Iterator kit = almaMaterSumCount.keySet().iterator();
		while (kit.hasNext()) {
			String k = (String) kit.next();
			String sn = peopleDB.getSchoolName(k, nameMap);
			if (sn != null) {
				if (!sn.equals(k)) {
					parentMap.put(k, sn);
				}
				continue;
			}
			// if the sn == null, try high school
			noNameList.add(new UEntry(k, almaMaterSumCount.get(k)));
		}
		/**
		 * kit = institutionSumCount.keySet().iterator(); while (kit.hasNext())
		 * { String k = (String)kit.next(); String sn = getSchoolName(k,
		 * nameMap); if (sn != null) { if (!sn.equals(k)) { parentMap.put(k,
		 * sn); } continue; } }
		 */
		Collections.sort(noNameList);
		printAlmamater(noNameList, dataDir + File.separator + "NoNameList.csv");

		// consolidate all schools that belong to the same school, e.g.,
		// consolidate "Harvard College" into "Harvard University"
		System.out.println("building consolidated almamater and institution lists");
		consolidateSchools(parentMap, almaMaterSumCount, almaMaterCount);
		consolidateSchools(parentMap, institutionSumCount, institutionCount);

		System.out.println("calculating almamater statistics");
		ArrayList<UEntry> clist = new ArrayList<UEntry>();
		Iterator cit = almaMaterSumCount.entrySet().iterator();
		while (cit.hasNext()) {
			Map.Entry pairs = (Map.Entry) cit.next();
			Integer v = (Integer) pairs.getValue();
			clist.add(new UEntry((String) pairs.getKey(), (double) v.intValue()));
		}
		System.out.println("sorting based on consolidated almamater statistics");
		Collections.sort(clist);
		System.out.println("printing consolidated almamater list ...");
		printAlmamater(clist, dataDir + File.separator + "AlmamaterConsolidatedList.csv");

		clist = new ArrayList<UEntry>();
		cit = institutionSumCount.entrySet().iterator();
		while (cit.hasNext()) {
			Map.Entry pairs = (Map.Entry) cit.next();
			Integer v = (Integer) pairs.getValue();
			clist.add(new UEntry((String) pairs.getKey(), (double) v.intValue()));
		}
		System.out.println("sorting based on consolidated institution statistics");
		Collections.sort(clist);
		System.out.println("printing consolidated institution list ...");
		printInstitution(clist, dataDir + File.separator + "InstitutionConsolidatedList.csv");

		peopleDB.close();
	}

	static void consolidateSchools(HashMap<String, String> parentMap, HashMap<String, Integer> sumMap,
			HashMap<String, Integer> countMap) {
		Iterator pkit = parentMap.keySet().iterator();
		while (pkit.hasNext()) {
			String k = (String) pkit.next();
			String v = parentMap.get(k);
			Integer scInteger = sumMap.get(k);
			if (scInteger == null)
				continue;
			int sc = scInteger.intValue();
			int c = countMap.get(k);
			sumMap.remove(k);
			countMap.remove(k);
			if (sumMap.get(v) == null) {
				sumMap.put(v, sc);
				countMap.put(v, c);
			} else {
				sumMap.put(v, (sumMap.get(v) + sc));
				countMap.put(v, (countMap.get(v) + c));
			}
		}
	}

	// get the real school name from a subsidiary name
	String getSchoolName(String sname, HashMap<String, String> nameMap) {
		if (nameMap.get(sname) != null)
			return sname;
		// if there is no entry in the nameMap, look up the parent categories
		String a = wikiDB.getArticleTitleByTitle(sname);
		if (a == null)
			return null;

		ArrayList<String> pl = wikiDB.getParentCategoryTitles(a, WikiDB.PageType.ARTICLE);
		if (pl == null)
			return null;
		for (int i = 0; i < pl.size(); i++) {
			String pn = pl.get(i);
			if (nameMap.get(pn) != null)
				return pn;
			// if there is no entry even in the parent, look up one level upper
			ArrayList<String> ppl = wikiDB.getParentCategoryTitles(pl.get(i), WikiDB.PageType.CATEGORY);
			for (int j = 0; j < ppl.size(); j++) {
				String pc = ppl.get(j);
				if (nameMap.get(pc) != null)
					return pc;
				// add another layer, example is JFK School of Government, has
				// two layers up to Harvard University
				ArrayList<String> pppl = wikiDB.getParentCategoryTitles(ppl.get(j), WikiDB.PageType.CATEGORY);
				for (int k = 0; k < pppl.size(); k++) {
					String ppc = pppl.get(k);
					if (nameMap.get(ppc) != null)
						return ppc;
				}
			}
		}
		return null;
	}

	static ArrayList<PersonEntry> readPersonList(WikiDB wikiDB, String filename) throws IOException {
		File file = new File(filename);
		return readPersonList(wikiDB, file);
	}

	static ArrayList<PersonEntry> readPersonList(WikiDB wikiDB, File file) throws IOException {
		ArrayList<PersonEntry> palist = new ArrayList<PersonEntry>();

		Reader reader = new InputStreamReader(new FileInputStream(file), "UTF-8");
		LineNumberReader lnr = new LineNumberReader(reader);
		// CSVReader csvReader = new CSVReader(reader);

		int i = 0;
		String[] row = null;
		schemaRow = DBpediaTable.parseCSV(lnr.readLine());

		int lineCount = 0;
		String lineString = null;
		while ((lineString = lnr.readLine()) != null) {
			lineCount++;
			row = DBpediaTable.parseCSV(lineString);
			if (row == null)
				continue;
			if (row.length <= 1)
				continue;
			if (row.length < schemaRow.length) {
				System.out.println("data does not comply with schema: " + row[1]);
				continue;
			}
			String ps = row[1].trim();

			if (ps == null || ps.equals("NULL"))
				continue;

			if (row[0].trim().equalsIgnoreCase("Uniform resource identifier"))
				continue;
			if (row[0].trim().equalsIgnoreCase("URI"))
				continue;
			if (ps.trim().equalsIgnoreCase("Uniform resource identifier"))
				continue;
			if (ps.trim().equalsIgnoreCase("URI"))
				continue;

			String a = DBpediaTable.getArticleTitle(ps, row[0]);
			if (a == null)
				continue;
			palist.add(new PersonEntry(wikiDB, a, row));
			i++;
			if (i % 10000 == 0)
				System.out.println(dateFormat.format(new Date()) + " read " + i + " people ...");
		}

		lnr.close();
		reader.close();
		return palist;
	}

	static void printAlmamater(ArrayList<UEntry> ulist, String filename) throws IOException {
		OutputStreamWriter fw = new OutputStreamWriter(new FileOutputStream(filename), "UTF-8");
		for (int i = 0; i < ulist.size(); i++) {
			UEntry ue = ulist.get(i);
			Integer sum = almaMaterSumCount.get(ue.schoolName);
			Integer count = almaMaterCount.get(ue.schoolName);
			fw.append("\"" + ue.schoolName + "\"," + sum + "," + count + "\n");
		}
		fw.close();
	}

	static void printInstitution(ArrayList<UEntry> ulist, String filename) throws IOException {
		OutputStreamWriter fw = new OutputStreamWriter(new FileOutputStream(filename), "UTF-8");
		for (int i = 0; i < ulist.size(); i++) {
			UEntry ue = ulist.get(i);
			Integer count = institutionCount.get(ue.schoolName);
			fw.append("\"" + ue.schoolName + "\"," + ue.score + "," + count + "\n");
		}
		fw.close();
	}

	/**
	 * from the Persons' almamater to calculate weight/score of school
	 * 
	 * @param palist
	 */
	static void schoolStat(ArrayList<PersonEntry> palist) {
		almaMaterSumCount = new HashMap<String, Integer>();
		almaMaterCount = new HashMap<String, Integer>();
		institutionSumCount = new HashMap<String, Integer>();
		institutionCount = new HashMap<String, Integer>();

		for (int i = 0; i < palist.size(); i++) {

			int count = palist.get(i).score + 1; // for the person who has an
													// almamater but his score
													// is 0, should still count
													// as at least 1

			String[] slist = palist.get(i).almamater;
			if (slist != null && slist.length > 0)
				for (int k = 0; k < slist.length; k++) {
					String s = slist[k];
					countSchool(almaMaterSumCount, almaMaterCount, s, count);
				}

			slist = palist.get(i).institution;
			if (slist != null && slist.length > 0)
				for (int k = 0; k < slist.length; k++) {
					String s = slist[k];
					countSchool(institutionSumCount, institutionCount, s, count);
				}
		}
		return;
	}

	static void countSchool(HashMap<String, Integer> sumMap, HashMap<String, Integer> countMap, String toBeAdded,
			int count) {
		if (!sumMap.containsKey(toBeAdded)) {
			sumMap.put(toBeAdded, Integer.valueOf(count));
			countMap.put(toBeAdded, Integer.valueOf(1));
		} else {
			Integer sc = sumMap.get(toBeAdded);
			Integer c = countMap.get(toBeAdded);
			sumMap.put(toBeAdded, Integer.valueOf(sc.intValue() + count));
			countMap.put(toBeAdded, Integer.valueOf(c.intValue() + 1));
		}
	}

	/**
	 * print the person list in ranking based on TotalLinksInCount
	 */
	static void printRankingDataInCSV(ArrayList<PersonEntry> palist, String filename) throws IOException {
		OutputStreamWriter fw = new OutputStreamWriter(new FileOutputStream(filename), "UTF-8");
		for (int i = 0; i < palist.size(); i++) {
			if (i % 10000 == 0)
				System.out.println(
						dateFormat.format(new Date()) + " printed " + i + " people ranking into file " + filename);
			String a = palist.get(i).a;
			String almamater = palist.get(i).getAlmamater();
			String institution = palist.get(i).getInstitution();

			int weightedLinksIn = 0;

			// ArrayList<String> linksin = wikiDB.getLinksIn(a);
			// if (linksin != null && linksin.size() != 0) for (int k = 0; k <
			// linksin.size(); k++) {weightedLinksIn +=
			// wikiDB.getTotalLinksInCount(linksin.get(k));}

			fw.write("\"" + a + "\"," + palist.get(i).score + "," + weightedLinksIn + ",\"" + almamater + "\",\""
					+ institution + "\"\n");
		}
		fw.close();
	}

	/**
	 * static List<PersonEntry> getPersonListFromArticleOutlinks(String
	 * articleTitle) { Article art=wikipedia.getArticleByTitle(articleTitle); if
	 * (art == null) {System.out.println("there is no article titled " +
	 * articleTitle); System.exit(0);} return getPersonList(art.getLinksOut());
	 * }
	 */
	public List<PersonEntry> getPersonListFromArticle(String articleTitle, int steps) {
		String art = wikiDB.getArticleTitleByTitle(articleTitle);
		if (art == null) {
			System.out.println("there is no article titled " + articleTitle);
			return null;
		}
		ArrayList<String> al = wikiDB.getArticleLinksOut(art, steps);

		return getPersonList(al);
	}

	public List<PersonEntry> getPersonList(ArrayList<String> al) {
		ArrayList<PersonEntry> plist = new ArrayList<PersonEntry>();
		for (int i = 0; i < al.size(); i++) {
			PersonEntry pe = name2person.get(al.get(i));
			if (pe != null) {
				plist.add(pe);
			} else {
				// System.out.println("can't find name: " + name + " in the
				// name2Person map");
			}
		}
		return plist;
	}

	public List<PersonEntry> getListOfAtheists() {
		return getPersonListFromArticle("List of atheists", 2);
	}

	public List<PersonEntry> getListOfPhysicists() {
		try {
			String filename = "peopleranking" + File.separator + "people" + File.separator + "PhysicistList.txt";
			List<PersonEntry> plist = getListFromTextFile(new File(filename), "—");
			String catname = "Physicists by nationality";
			List<PersonEntry> list2 = getPersonListByCategory(catname, 4);
			for (int i = 0; i < list2.size(); i++) {
				if (plist.contains(list2.get(i)))
					continue;
				plist.add(list2.get(i));
			}
			list2 = getPersonListFromArticle("List of physicists", 1);
			for (int i = 0; i < list2.size(); i++) {
				if (plist.contains(list2.get(i)))
					continue;
				plist.add(list2.get(i));
			}
			return plist;
		} catch (IOException ioe) {
		}

		return null;
	}

	public List<PersonEntry> getListOfComputerScientists() {
		String artname = "List of computer scientists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		List<PersonEntry> list2 = getPersonListFromArticle("List of programmers", 1);
		for (int i = 0; i < list2.size(); i++) {
			if (plist.contains(list2.get(i)))
				continue;
			plist.add(list2.get(i));
		}
		return plist;
	}

	public List<PersonEntry> getListOfLogicians() {
		String artname = "List of logicians";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfPsychologists() {
		String artname = "List of psychologists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfLinguists() {
		String artname = "List of linguists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfSociologists() {
		String artname = "List of sociologists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfHistorians() {
		String artname = "List of historians";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfEconomists() {
		String artname = "List of economists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfPoliticalScientists() {
		String artname = "List of political scientists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfEngineers() {
		String artname = "List of engineers";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 2);
		return plist;
	}

	public List<PersonEntry> getListOfMechanicalEngineers() {
		String artname = "List of mechanical engineers";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfChemicalEngineers() {
		String artname = "List of chemical engineers";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfCivilEngineers() {
		String artname = "List of civil engineers";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfElectricalEngineers() {
		String artname = "List of electrical engineers";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfLibrarians() {
		String artname = "List of librarians";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 1);
		return plist;
	}

	public List<PersonEntry> getListOfJournalists() {
		String artname = "List of journalists";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 2);
		return plist;
	}

	public List<PersonEntry> getListOfWriters() {
		String artname = "List of writers";
		List<PersonEntry> plist = getPersonListFromArticle(artname, 2);
		return plist;
	}

	public List<PersonEntry> getListOfChessPlayers() {
		String filename = "peopleranking" + File.separator + "people" + File.separator + "ChessPlayer.csv";
		List<PersonEntry> plist = null;
		try {
			plist = readPersonList(wikiDB, filename);
		} catch (IOException ioe) {
			return plist;
		}
		return plist;
	}

	public List<PersonEntry> getListOfChefs() {
		String filename = "peopleranking" + File.separator + "people" + File.separator + "Chef.csv";
		List<PersonEntry> plist = null;
		try {
			plist = readPersonList(wikiDB, filename);
		} catch (IOException ioe) {
			return plist;
		}
		return plist;
	}

	public List<PersonEntry> getListOfPhilosophers() {
		String filename = "peopleranking" + File.separator + "people" + File.separator + "Philosopher.csv";
		List<PersonEntry> plist = null;
		try {
			plist = readPersonList(wikiDB, filename);
		} catch (IOException ioe) {
			return plist;
		}
		return plist;
	}

	public List<PersonEntry> getListOfArchitects() {
		String filename = "peopleranking" + File.separator + "people" + File.separator + "Architect.csv";
		List<PersonEntry> plist = null;
		try {
			plist = readPersonList(wikiDB, filename);
		} catch (IOException ioe) {
			return plist;
		}
		return plist;
	}

	public List<PersonEntry> getListOfSpies() {
		String catname = "Spies by nationality";
		return getPersonListByCategory(catname, 2);
	}

	public List<PersonEntry> getListOfActors() {
		String catname = "Actors";
		return getPersonListByCategory(catname, 5);
	}

	public List<PersonEntry> getListOfCEOs() {
		String catname = "Chief executive officers";
		return getPersonListByCategory(catname, 4);
	}

	public List<PersonEntry> getListOfLGBTs() {
		String catname = "LGBT people by nationality";
		return getPersonListByCategory(catname, 4);
	}

	public List<PersonEntry> getPersonListByCategory(String categoryTitle, int level) {
		ArrayList<String> alist = wikiDB.getDescendentArticleTitles(categoryTitle, level);
		return getPersonList(alist);
	}

	public List<PersonEntry> getListFromTable(CSVTable tab, String columnName) {
		List<String> namelist = getNameListFromTable(tab, columnName);
		if (namelist == null || namelist.size() == 0)
			return null;
		ArrayList<String> al = new ArrayList<String>();
		for (int i = 0; i < namelist.size(); i++) {
			String name = namelist.get(i);
			if (name.equals("—") || name.equalsIgnoreCase("NULL") || name.equalsIgnoreCase("None"))
				continue;
			String pn = namelist.get(i).trim();
			String a = wikiDB.getArticleTitleByTitle(pn);
			if (a != null)
				al.add(a);
			else
				System.out.println("cannot find " + namelist.get(i) + " in wikipedia");
		}

		return getPersonList(al);
	}

	public List<PersonEntry> getListOfNovelists() {
		String artname = "List of novelists by nationality";
		return getPersonListFromArticle(artname, 1);
	}

	public List<PersonEntry> getListOfBiologists() {
		String listname = "List of biologists";
		return getPersonListFromArticle(listname, 1);
	}

	public List<PersonEntry> getListOfPoets() {
		String listname = "List of poets";
		return getPersonListFromArticle(listname, 1);
	}

	public List<PersonEntry> getListOfChemists() {
		String listname = "List of chemists";
		return getPersonListFromArticle(listname, 1);
	}

	public List<PersonEntry> getListOfMathematicians() {
		try {
			String filename = "peopleranking" + File.separator + "people" + File.separator + "MathematicianList.txt";
			List<PersonEntry> plist = getListFromTextFile(new File(filename), "(");
			List<PersonEntry> alist = getPersonListFromArticle("List of mathematicians", 1);
			System.out.println(
					"from wiki list mathematicians, there are " + plist.size() + " mathematicians in the list");
			for (int i = 0; i < alist.size(); i++) {
				if (plist.contains(alist.get(i)))
					continue;

				plist.add(alist.get(i));
			}
			return plist;
		} catch (IOException ioe) {

		}
		return null;
	}

	/*
	 * get list from a txt file with essentially a list of names separated by
	 * linefeed
	 */
	public List<PersonEntry> getListFromTextFile(File f, String sep) throws IOException {
		LineNumberReader lineReader = new LineNumberReader(
				new java.io.InputStreamReader(new java.io.FileInputStream(f), "UTF-8"));
		String row = null;
		List<PersonEntry> palist = new ArrayList<PersonEntry>();
		int i = 0;
		while ((row = lineReader.readLine()) != null) {
			int si = row.indexOf(sep);
			if (si <= 0)
				continue;
			row = row.substring(0, si).trim();

			PersonEntry pe = name2person.get(row);
			if (pe == null) {
				int commaind = row.indexOf(",");
				if (commaind <= 0)
					continue;
				String sec1 = row.substring(0, commaind).trim();
				String sec2 = row.substring(commaind + 1).trim();
				String ns = sec2 + " " + sec1;
				pe = name2person.get(ns);
				if (pe == null)
					continue;
			}

			palist.add(pe);

			i++;
			if (i % 1000 == 0)
				System.out.println(dateFormat.format(new Date()) + " read " + i + " people ...");
		}

		lineReader.close();
		System.out.println(f.getName() + " has " + i + " rows" + " and palist has " + palist.size() + " entries");

		return palist;
	}

	public List<String> getNameListFromTable(CSVTable tab, String columnName) {

		List<String> nameList = tab.getColumn(columnName);
		if (nameList == null || nameList.size() == 0)
			return null;

		ArrayList<String> retList = new ArrayList<String>(nameList.size());

		for (int i = 0; i < nameList.size(); i++) {
			String nameStr = nameList.get(i);
			String[] namea;
			if (nameStr.contains(";")) {
				namea = nameStr.split(";");
			} else {
				namea = new String[1];
				namea[0] = nameStr;
			}
			for (int k = 0; k < namea.length; k++) {
				retList.add(namea[k].trim());
			}
		}
		System.out.println("the length of the " + columnName + " list is " + retList.size());
		return retList;

	}

	public ArrayList<PersonEntry> getNobelLaureatesNoPeace() throws IOException {
		String filename = "peopleranking" + File.separator + "nobellist.csv";
		CSVTable ctable = new CSVTable(filename);
		System.out.println("there are " + ctable.table.size() + " rows, and " + ctable.schemaRow.length + " columns");

		ArrayList<PersonEntry> allList = new ArrayList<PersonEntry>();
		String[] namerow = { "Physics", "Chemistry", "Physiology or Medicine", "Literature", "Peace", "Economics" };
		for (int i = 0; i < namerow.length; i++) {
			if (i == 4)
				continue;
			String name = namerow[i];
			System.out.println("processing " + name + " list");
			List<PersonEntry> pl = getListFromTable(ctable, name);
			// System.out.println("there are " + pl.size() + " in the " + name +
			// " list");
			// printZodiacCount(pl);
			allList.addAll(pl);
		}
		return allList;
	}

}

class PersonEntry {
	String a;
	String[] almamater;
	String[] institution;
	String birthDate = null;
	String deathDate = null;
	String gender = null;
	// static WikiDB wikiDB;
	int score;

	public PersonEntry(WikiDB wikiDB, String a, String[] row) { // String
																// almamater,
																// String
																// institution)
																// {
		this.a = a;
		// this.wikiDB = wikiDB;
		System.out.println(a);
		score = wikiDB.getTotalLinksInCount(a);
		int almaMaterIndex = getSchemaIndex("almaMater_label");
		int institutionIndex = getSchemaIndex("institution_label");
		int almaMaterResourceIndex = getSchemaIndex("almaMater");
		int institutionResourceIndex = getSchemaIndex("institution");
		int birthDateIndex = getSchemaIndex("birthDate");
		int deathDateIndex = getSchemaIndex("deathDate");
		int genderIndex = getSchemaIndex("gender_label");

		String almaMaterStr = null;
		if (almaMaterIndex >= 0)
			almaMaterStr = row[almaMaterIndex];
		String institutionStr = null;
		if (institutionIndex >= 0)
			institutionStr = row[institutionIndex];

		String birthDateStr = null;
		if (birthDateIndex >= 0)
			birthDateStr = row[birthDateIndex];
		String deathDateStr = null;
		if (deathDateIndex >= 0)
			deathDateStr = row[deathDateIndex];

		String genderstr = null;
		if (genderIndex >= 0)
			genderstr = row[genderIndex];
		if (genderstr != null && !genderstr.equalsIgnoreCase("NULL"))
			gender = genderstr.trim();
		this.almamater = parseWikibag(almaMaterStr);
		this.institution = parseWikibag(institutionStr);
		if (birthDateStr != null && birthDateStr.trim().length() > 0 && !birthDateStr.equalsIgnoreCase("NULL"))
			this.birthDate = birthDateStr.trim();// DBpediaTable.parseDate(dateStr);
		if (deathDateStr != null && deathDateStr.trim().length() > 0 && !deathDateStr.equalsIgnoreCase("NULL"))
			this.deathDate = deathDateStr.trim();// DBpediaTable.parseDate(dateStr);

		String almaMaterResourceStr = null;
		if (almaMaterResourceIndex >= 0)
			almaMaterResourceStr = row[almaMaterResourceIndex];
		String institutionResourceStr = null;
		if (institutionResourceIndex >= 0)
			institutionResourceStr = row[institutionResourceIndex];
		String[] almaMaterResource = parseWikibag(almaMaterResourceStr);
		String[] institutionResource = parseWikibag(institutionResourceStr);

		if (almamater != null && almaMaterResource != null) {
			int lim = Math.min(almamater.length, almaMaterResource.length);
			for (int i = 0; i < lim; i++) {
				String almaMaterArticle = DBpediaTable.getArticleTitle(almamater[i], almaMaterResource[i]);
				if (almaMaterArticle != null)
					almamater[i] = almaMaterArticle;
			}
		}
		if (institution != null && institutionResource != null) {
			int lim = Math.min(institution.length, institutionResource.length);
			for (int i = 0; i < lim; i++) {
				String institutionArticle = DBpediaTable.getArticleTitle(institution[i], institutionResource[i]);
				if (institutionArticle != null)
					institution[i] = institutionArticle;
			}
		}
	}

	static int getSchemaIndex(String propertyName) {
		for (int i = 0; i < PeopleDB.schemaRow.length; i++) {
			if (PeopleDB.schemaRow[i].equals(propertyName))
				return i;
		}
		return -1;
	}

	String[] parseWikibag(String ams) {
		if (ams == null)
			return null;
		String s = ams.trim();
		if (s.equals("NULL"))
			return null;
		if (s.startsWith("{")) {
			if (!s.endsWith("}"))
				return null;
			s = s.substring(1, s.length() - 1).trim();
			return s.split("\\|");
		}
		String[] ret = new String[1];
		ret[0] = s;
		return ret;
	}

	String getAlmamater() {
		return getWikibag(almamater);
	}

	String getInstitution() {
		return getWikibag(institution);
	}

	String getWikibag(String[] bag) {
		if (bag == null)
			return "NULL";
		String ret = "{";
		for (int i = 0; i < bag.length; i++) {
			ret += bag[i];
			if (i < (bag.length - 1))
				ret += "|";
		}
		ret += "}";
		return ret;
	}

	/**
	 * return true if the corresponding articles equals ths other one's
	 * 
	 */
	public boolean equals(Object o) {
		if (o instanceof PersonEntry) {
			PersonEntry p = (PersonEntry) o;
			return (p.a.equals(this.a));
		}
		return false;
	}

}

class PersonComparator implements Comparator<PersonEntry> {
	public int compare(PersonEntry p1, PersonEntry p2) {
		return (p2.score - p1.score);
	}
}
